#! /usr/local/bin/ruby

require 'find'
require 'optparse'
require 'ostruct'
require 'generator'
require 'yaml'
autoload :StringIO, 'stringio.rb'

require File.dirname(__FILE__) + '/text_file'
require File.dirname(__FILE__) + '/files_by_digest'
require File.dirname(__FILE__) + '/groups_of_files'
require File.dirname(__FILE__) + '/prefs'
require File.dirname(__FILE__) + '/canonizer'
require File.dirname(__FILE__) + '/ui'
require File.dirname(__FILE__) + '/document'
require File.dirname(__FILE__) + '/suffix'
require File.dirname(__FILE__) + '/file_based_document'
require File.dirname(__FILE__) + '/dup_finder'

class DupDetector
  # directories_to_search can be a string or a list of strings.

  # ignoredFiles consists of a list of filenames, pathnames, and directories.
  #
  # A filename (the string contains no slash) will be ignored in any directory
  # in which it appears.
  #
  # A pathname (the string contains a slash and resolves to a file) will be
  # ignored only if the complete path to the file, starting from the directory
  # in directories_to_search, contains the string.
  #
  # A directory (the string contains a slash and resolves to a directory)
  # will be ignored along with all of its contents.

  # If extensions is nil then all files will be checked.
  # Otherwise extensions contains a list of strings, and only files whose
  # names end with those strings will be checked.
  def initialize(directories_to_search, ignoredFiles = nil, extensions = nil, use_yaml_prefs = false)
    if directories_to_search.is_a?(String)
      @directories_to_search = [directories_to_search]
    else
      @directories_to_search = directories_to_search
    end

    if ignoredFiles.nil?
      if use_yaml_prefs
        prefs = Prefs.read()
        ignoredFiles = prefs['ignore_list']
      end
      ignoredFiles = [] if ignoredFiles.nil?
    end

    # Select all strings in the array which contain at least one slash.
    @ignoredPaths = ignoredFiles.select { | entry | %r'/' =~ entry }
    @ignoredPaths.map! {|entry| entry.chomp('/') }

    # The complement of the set above.
    @ignoredFilenames = ignoredFiles - @ignoredPaths

    @extensions_to_include = extensions

    if extensions.nil?
      prefs = Prefs.read() if use_yaml_prefs
      @extensions_to_include = prefs['extensions'] if use_yaml_prefs
    end

    # There's more than one way to do it. This would also work:
    # @ignoredFilenames = ignoredFiles.reject { | entry | %r'/' =~ entry }
  end

  def extension_in_list?(f)
    return true if @extensions_to_include.nil?

    @extensions_to_include.each do |extension|
      return true if f =~ /\.#{extension}$/
    end

    return false
  end

  def find_qualifying_files()
    paths = []

    Find.find(*@directories_to_search) do |path|
      Find.prune() if path =~ /\.svn/

      if File.directory?(path) && @ignoredPaths.include?(path)
        Find.prune()
      end

      unless File.directory?(path)
        paths << path unless
          @ignoredFilenames.include?(File::basename(path)) ||
          @ignoredPaths.include?(path) ||
          !extension_in_list?(path)
      end
    end

    paths
  end

  def get_dups_by_criteria(&block)
    paths = {}

    find_qualifying_files().each do | path |
      keys = yield(path)

      keys.each do |key|
        paths[key] ||= []
        paths[key] << path
      end
    end

    GroupsOfFiles.new(paths)
  end

  def self.file_in_any_of_directories(filepath, dirs)
    dirs.detect { | dir | filepath.index(dir) == 0 }
  end

  def self.select_dirs_from_paths(paths)
    paths.select { | entry | entry =~ /.+\/$/ }
  end

  def self.extract_article_name s
    s = File::basename(s, '.*') # Remove the suffix, if any

    m = /\d+ +[^ ]+ +[^ ]+ +([^ ].*)/.match(s)

    article_name = m[1]

    article_name_sans_spaces = article_name.gsub(/\s/, "") # Remove all whitespace

    Canonizer.canonize(article_name_sans_spaces)
  end
end

class BasenamesDupDetector < DupDetector
  def get_dups
    get_dups_by_criteria { |path| [File::basename(path, '.*')] }
  end
end

class ContentsDupDetector < DupDetector
  def get_dups
    get_dups_by_criteria { |path| [Digest::SHA1.hexdigest(File.read(path))] }
  end
end

class TextDupDetector < DupDetector
  def get_dups
    get_dups_by_criteria { |path| [File.read(path)] }
  end
end

class InternalDupDetector < DupDetector
  def initialize(min_length, directories_to_search, ignoredFiles = nil, extensions = nil)
    super(directories_to_search, ignoredFiles, extensions)

    @min_length = min_length
  end
  
  def get_dups
    gen = Generator.new do |result|
      find_qualifying_files().each do | path |
        content = File.read(path)     
        result.yield([path, content])
      end
    end
     
    DupFinder.new(@min_length).get_internal_dups(gen)
  end

end


def parse_command_line(argv)
  options = OpenStruct.new

  opts = OptionParser.new do |opts|

    opts.banner = "dup_detector.rb option tree_to_search [another_tree ...]"
    opts.separator ""
    opts.separator "Specify exactly one of --basenames or --contents or --internal:"

    opts.on("-b", "--basenames", "Look only for files with same basename.") do |flag|
      options.basenames = flag
    end

    opts.on("-c", "--contents", "Look only for files with bit for bit identical contents.") do |flag|
      options.contents = flag
    end

    options.internal = false
    opts.on("-i", "--internal DUP_COUNT", Integer,
            "Look for files containing internal duplicates of at least DUP_COUNT words.") do |dup_count|
      options.dup_count = dup_count
      options.internal = true
    end

    opts.separator ''

    opts.on('-h', '--help', 'Show this message') do |flag|
      puts opts
      exit
    end
  end

  # Eat argv elements while parsing.
  opts.parse!(argv)

  flag_count = 0
  flag_count += 1 if options.basenames
  flag_count += 1 if options.contents
  flag_count += 1 if options.internal
  unless flag_count == 1
    puts opts
    exit
  end

  options
end

def main(argv)
  options = parse_command_line(argv)

  # Output is in the same format as the input, i.e.,
  # commandline input produces commandline output and
  # gui input produces gui output.

  if argv.empty?  # no specified directories to search
    require 'java'
    UI.get_gui_input
    groups_of_files = ContentsDupDetector.new([]).get_dups
    UI.set_gui_output(groups_of_files)
    ""
  else
    directories_to_search = argv

    if options.basenames
      dd = BasenamesDupDetector.new(directories_to_search)
    elsif options.contents
      dd = ContentsDupDetector.new(directories_to_search)
    elsif options.internal
      dd = InternalDupDetector.new(options.dup_count, directories_to_search)
    end

    dd.get_dups.to_table
  end
end

if __FILE__ == $0
  sio = StringIO.new(main(ARGV), 'r')
  sio.each_line { |line| puts line }
end
